/*
 * Copyright 2008 FBK (http://www.fbk.eu/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.fbk.it.hlt.jlsi.data;

import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.text.BreakIterator;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;

import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.fbk.it.hlt.jlsi.DocumentIndex;
import org.fbk.it.hlt.jlsi.TermIndex;
import org.fbk.it.hlt.jlsi.Vocabulary;
import org.fbk.it.hlt.jlsi.util.FolderScanner;
import org.fbk.it.hlt.jlsi.util.TextFilter;
import org.fbk.it.hlt.jlsi.util.WordSet;

/**
 * Create a term-by-document matrix in sparse binary format. Takes as input the
 * competitor analisys corpus.
 * 
 * @author Claudio Giuliano
 * @version %I%, %G%
 * @since 1.0
 */
public class CompetitorAnalysisCorpusReader {
	/**
	 * Define a static logger variable so that it references the Logger instance
	 * named <code>CompetitorAnalysisCorpusReader</code>.
	 */
	static Logger logger = Logger
			.getLogger(CompetitorAnalysisCorpusReader.class.getName());

	/**
	 * The term index
	 */
	private TermIndex termIndex;

	/**
	 * The document index
	 */
	private DocumentIndex documentIndex;

	/**
	 * The matrix writer.
	 */
	private MatrixFileWriter matrixWriter;

	//
	private int columnCount;

	//
	Vocabulary corpusVocabulary;

	//
	WordSet stopwords;

	/**
	 * Constructs a corpus readDocumentLister.
	 */
	public CompetitorAnalysisCorpusReader(File root, String matrixName,
			File stop, int n) {
		try {
			stopwords = new WordSet();
			stopwords.read(new FileReader(stop));

			columnCount = 0;

			File matrixFile = new File(matrixName + "-matrix");
			File rowFile = new File(matrixName + "-row");
			File colFile = new File(matrixName + "-col");
			File dfFile = new File(matrixName + "-df");

			FolderScanner fs = new FolderScanner(root);
			fs.setFiler(new TextFilter());

			termIndex = TermIndex.getTermIndex();
			documentIndex = DocumentIndex.getDocumentIndex();
			matrixWriter = new SparseBinaryMatrixFileWriter(matrixFile);
			corpusVocabulary = new Vocabulary();

			int count = 1;
			while (fs.hasNext()) {
				Object[] files = fs.next();
				// logger.debug((count++) + " : " + files.length);

				for (int i = 0; i < files.length; i++) {
					long begin = System.currentTimeMillis();
					// logger.debug((File) files[i]);
					Document d = new CompetitorAnalysisDocument((File) files[i]);
					logger.debug(d.title() + " " + d.id());
					readDocument(d);
					long end = System.currentTimeMillis();

					logger.info(files[i] + " processed in " + (end - begin)
							+ " ms");

				} // end for i

				count++;

				// if (count > n)
				// break;

			} // end while

			//
			termIndex.write(new FileWriter(rowFile));

			//
			documentIndex.write(new FileWriter(colFile));

			//
			matrixWriter.close();

			//
			corpusVocabulary.write(new FileWriter(dfFile));

			logger.info("columnCount: " + columnCount);
		} catch (Exception e) {
			logger.error(e);
		}
	} // end constructor

	//
	private void readDocument(Document d) throws IOException {
		// logger.debug("readSentence");
		Vocabulary documentVocabulary = new Vocabulary();

		int documentID = documentIndex.get(d.id());
		// /System.out.print(documentID + " \"" + sent + "\"\n");

		// iterates over the tokens
		// logger.debug("iterates over the tokens");
		BreakIterator boundary = BreakIterator.getWordInstance(Locale.US);
		boundary.setText(d.text());
		int start = boundary.first();
		for (int end = boundary.next(); end != BreakIterator.DONE; start = end, end = boundary
				.next()) {
			String token = d.text().substring(start, end).toLowerCase();
			// System.out.println(token);
			if (token.length() > 0
					&& !token.matches("\\s+")
					&& !stopwords.contains(token)
					&&
					// !token.equals("\t") && !token.equals("\n") &&
					!token.equals(".") && !token.equals(";")
					&& !token.equals(":") && !token.equals(",")
					&& !token.equals("!") && !token.equals("?")
					&& !token.equals("\"") && !token.equals("")
					&& !token.equals("(") && !token.equals(")")) {
				documentVocabulary.add(token);
			}
		} // end for i

		int size = documentVocabulary.entrySet().size();

		int[] indexes = new int[size];
		float[] values = new float[size];
		int j = 0;

		// iterates over the types
		// logger.debug("iterates over the types");
		Iterator it = documentVocabulary.entrySet().iterator();
		while (it.hasNext()) {
			Map.Entry me = (Map.Entry) it.next();
			String term = (String) me.getKey();
			Vocabulary.TermFrequency tf = (Vocabulary.TermFrequency) me
					.getValue();

			int ti = termIndex.put(term);
			indexes[j] = ti;
			// values[j] = (float) (tf.get() / size);

			// ???? (1 + log (tf) ????
			// values[j] = (float) tf.get();
			values[j] = (float) (1 + Math.log(tf.get()));
			// /logger.debug(term + " " + ti + " " + documentID + " (" + tf +
			// ")");

			corpusVocabulary.add(term);
			j++;
		} // end while

		columnCount++;
		matrixWriter.writeColumn(indexes, values);

	} // end readSentence

	//
	public static void main(String[] args) throws Exception {
		String logConfig = System.getProperty("log-config");
		if (logConfig == null)
			logConfig = "log-config.txt";

		long begin = System.currentTimeMillis();

		PropertyConfigurator.configure(logConfig);

		if (args.length != 4) {
			System.out
					.println("Usage: java -mx1024M org.fbk.it.hlt.jlsi.data.CompetitorAnalysisCorpusReader corpus-root term-document-file stopwords-file n");
			System.exit(1);
		}

		File root = new File(args[0]);
		String matrixName = args[1];
		File stop = new File(args[2]);
		int n = Integer.parseInt(args[3]);
		new CompetitorAnalysisCorpusReader(root, matrixName, stop, n);

		long end = System.currentTimeMillis();
		System.out.println("corpus readDocumentList in " + (end - begin)
				+ " ms");
	} // end main

} // end CompetitorAnalysisCorpusReader